var request = require('request');
var async = require('async');
var Iconv = require('iconv-lite');
var url_node = require('url');


var default_setting = {
    timeOuts: 0,
    start: {
        url: 'http://www.1kkk.com/manhua-jpkr-p*PageReplace*/',
        pageItem: 1,
        curPage: 1,
        totalPage: 4000,
        timeout: 5000
    },
    headers: {},
    mongodbParam: {},
    pageNext: function (default_setting) {

        // loop
        default_setting.start.curPage++;

        var url = default_setting.start.url.replace('*PageReplace*', default_setting.start.curPage);

        

        spider.spider(url, default_setting);
    },
    timeoutHandel: function (error, url, _default) {
        console.log('timeoutHandel');
        console.log(url);
        setTimeout(() => {
            spider.spider(url, _default, 999999);
        }, 3000);


    },

    successHandel: function (data) {

    },

    finishHandel: function (_default) {
        if (_default.start.curPage > _default.start.totalPage) {
            console.log('---spider all list---');

            _default.callback(null, "succ");
            spider = null;
         
            return true;
        } else {
            return false;
        }
    }
};

var spider = {};


spider.init = function (setting) {

    default_setting = extend(default_setting, setting);

    var url = default_setting.start.url.replace('*PageReplace*', default_setting.start.curPage);
    console.log("--------------------------");
    console.log("start:" + url);
    spider.spider(url, default_setting);

};

// url,setting,timeout
spider.spider = function () {

    var url = arguments[0];
    var _default = arguments[1];
    var timeout = arguments[2] || _default.start.timeout;
    //is finish
    if (_default.finishHandel(_default)) {

        return;
    }

    console.log("spider url:" + url);

    // url request
    request({ url: url, headers: _default.headers, encoding: null, timeout: timeout }, function (err, res, body) {
        // 超时错误处理
        if (err) {

            _default.timeoutHandel(err, url, _default);
            return;
        }       

        // body buffer
        _default.parseHandel(body, url, _default);
        // console.log(body.toString('utf-8'));
        body = null;

    });
};

spider.spiderDetail = function () {

    var url = arguments[0];
    var _default = arguments[1];
    var timeout = arguments[2] || _default.start.timeout;
    var t = arguments[3];

    var times = arguments[4] || 0;

    // console.log("spider url:" + url);

    // url request
    request({ url: url, headers: _default.headers, encoding: null, timeout: timeout }, function (err, res, body) {


        // 超时错误处理
        if (err) {
            console.log('spiderDetail timeoutHandel');
            console.log("spider url:" + url);
            console.log(times);

            console.log(t);
            // console.log(err);

            setTimeout(() => {
                if (times < 6) {

                    times++;

                    spider.spiderDetail(url, _default, 900000, t, times);


                } else {
                    times = null;
                    return;

                }
            }, 2000);

            return;

        }

        // body buffer
        _default.parseDetailHandel(body, url, _default, t);

    });
};

function extend() {
    var options, name, src, copy, copyIsArray, clone, target = arguments[0] || {},
        i = 1,
        length = arguments.length,
        deep = false,
        toString = Object.prototype.toString,
        hasOwn = Object.prototype.hasOwnProperty,
        push = Array.prototype.push,
        slice = Array.prototype.slice,
        trim = String.prototype.trim,
        indexOf = Array.prototype.indexOf,
        class2type = {
            "[object Boolean]": "boolean",
            "[object Number]": "number",
            "[object String]": "string",
            "[object Function]": "function",
            "[object Array]": "array",
            "[object Date]": "date",
            "[object RegExp]": "regexp",
            "[object Object]": "object"
        },
        jQuery = {
            isFunction: function (obj) {
                return jQuery.type(obj) === "function"
            },
            isArray: Array.isArray ||
                function (obj) {
                    return jQuery.type(obj) === "array"
                },
            isWindow: function (obj) {
                return obj != null && obj == obj.window
            },
            isNumeric: function (obj) {
                return !isNaN(parseFloat(obj)) && isFinite(obj)
            },
            type: function (obj) {
                return obj == null ? String(obj) : class2type[toString.call(obj)] || "object"
            },
            isPlainObject: function (obj) {
                if (!obj || jQuery.type(obj) !== "object" || obj.nodeType) {
                    return false
                }
                try {
                    if (obj.constructor && !hasOwn.call(obj, "constructor") && !hasOwn.call(obj.constructor.prototype, "isPrototypeOf")) {
                        return false
                    }
                } catch (e) {
                    return false
                }
                var key;
                for (key in obj) { }
                return key === undefined || hasOwn.call(obj, key)
            }
        };
    if (typeof target === "boolean") {
        deep = target;
        target = arguments[1] || {};
        i = 2;
    }
    if (typeof target !== "object" && !jQuery.isFunction(target)) {
        target = {}
    }
    if (length === i) {
        target = this;
        --i;
    }
    for (i; i < length; i++) {
        if ((options = arguments[i]) != null) {
            for (name in options) {
                src = target[name];
                copy = options[name];
                if (target === copy) {
                    continue
                }
                if (deep && copy && (jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)))) {
                    if (copyIsArray) {
                        copyIsArray = false;
                        clone = src && jQuery.isArray(src) ? src : []
                    } else {
                        clone = src && jQuery.isPlainObject(src) ? src : {};
                    }
                    // WARNING: RECURSION
                    target[name] = extend(deep, clone, copy);
                } else if (copy !== undefined) {
                    target[name] = copy;
                }
            }
        }
    }
    return target;
}

module.exports = Object.assign({},spider);